import json
import random
import re
import time
import requests
import pandas
import sys

class MeituanSpider():
    def __init__(self):
        # 入口url
        self.start_url_wx = 'https://wx-shangou.meituan.com/mtweapp/v1/poi/sputag/products?ui=582315343&region_id=1000330200&region_version=1734923445566&yodaReady=wx&csecappid=wxde8ac0a21135c07d&csecplatform=3&csecversionname=8.51.2&csecversion=1.4.0'
        self.start_url_h5 = 'https://wx-shangou.meituan.com/quickbuy/v1/poi/sputag/products?ui=582315343&region_id=1000330200&region_version=1734923445566&yodaReady=wx&csecappid=wxde8ac0a21135c07d&csecplatform=3&csecversionname=8.51.2&csecversion=1.4.0'

        # 首先需要登录自己的账号上 获取登录后的Cookie信息和User-Agent来构造响应头
        self.headers_h5 = {
            # 修改成自己的cookie
            "Cookie": "_lxsdk_cuid=193f1a01405c8-047f8ec4a9f501-28653612-42680-193f1a01405c8; bussiness_entry_channel=default; WEBDFPID=z0x2y0900z655968zyw96v2u2144y736806u8z9652577958682v63v2-2050285628340-1734925620200GMGAQMAfd79fef3d01d5e9aadc18ccd4d0c95072881; iuuid=97CAA3F4C5274B58E0BC0D926FF86136597DAE423E2B22E6A57436A9DE1093BA; token=AgGkJOtSBVxvex6lePNATYAkM4jnLGZqwHN1pPBMIk7ipTkvcDpm9W36cC2K71IvQJuyzaeFOShPYQAAAABrJQAA4aHeb5SSFQ2wkMV0FpfgocL514jeUoJg1sBBQGSU0-kp6IwBV3H-QgMe2yPwt2Ci; mt_c_token=AgGkJOtSBVxvex6lePNATYAkM4jnLGZqwHN1pPBMIk7ipTkvcDpm9W36cC2K71IvQJuyzaeFOShPYQAAAABrJQAA4aHeb5SSFQ2wkMV0FpfgocL514jeUoJg1sBBQGSU0-kp6IwBV3H-QgMe2yPwt2Ci; oops=AgGkJOtSBVxvex6lePNATYAkM4jnLGZqwHN1pPBMIk7ipTkvcDpm9W36cC2K71IvQJuyzaeFOShPYQAAAABrJQAA4aHeb5SSFQ2wkMV0FpfgocL514jeUoJg1sBBQGSU0-kp6IwBV3H-QgMe2yPwt2Ci; userId=582315343; _lxsdk_s=193f2606b11-47b-7ee-3c4%7C%7C765",
            "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36 Edg/131.0.0.0",
            "Content-Type": "application/x-www-form-urlencoded",
            "Mtgsig": '{"a1":"1.2","a2":1735007086938,"a3":"ww703v76769x5v44zvyw14y2z8003y6z806u82vz8w797978822z6w17","a4":"c5c7fa26d02d9c7826fac7c5789c2dd07cd3bc33044f91e3","a5":"K1/z2PqLfKK92USyHHUnlsZ7U11rnCZUptXMUNKNQ6cu1nZCW5LMw+HzCWCe5noEypPp84k77MwsYlY9KhbgvgNbAqtQlpYQRpsjtg39UWEbXCMBvbQLwNumZcxj3+pk0qH7A9CPJjihpfus2An8qJ2n8rLtj4NMs/L98xS1p2POvQEBb7HikBbR5Wt201G5360J5gwHGmIz8IYMnkpa/9NcvLq57tNT/usMf2EdQgwl7KhmiPP=","a6":"w1.3r8F+8MAmLFPCEm4z+uU77C/tKC1KTleTlhdtBi7fVjItSXlMC6RdveukvY/kCmsQRoBTXB8jtcngv7ORi7U/Q9SI8Faozi+1iuSO6ywOAfgIcE0oSvHBllY9tzhWnUzj3MsDM5tc4VGwG2GzbTnODQfJeyT9xN4U42IcvvdfmwewPLUluyGm4ghNOpt4nyT4ySYrBoftsje56/5sj6GIIz6XBTdxpHC3qkOiC7xEFsi5QEncPQObdErLF1oIwCr3txLAzE7rdzegjRYQFEyitJiWVTcfSrk3XxsQIf01589qMJ/gwhNgGCSeoSDsMRvls82/VQW1KmQd6H7XsLbgrNTN3TEWP1v+uJohjEK1EE/m/0sumswptQi96VFl61Rr4tIgzRvpLMgr+Q3xAy1FDno9DzGOamHQYEp2NzavXGF1Oiic/jJmkqKKA3CD2ek6gRrjIPcnN9MxgoDxyEtzbx02agvklGdOoWhqCKx04Tn3wyVp7z7Ht0tYSWBBrmxx","a7":"wxde8ac0a21135c07d","x0":3,"d1":"cfa3104bc35d4040f678cf9ea8d7e044"}'
        }

        self.data_h5 = {
            'wm_appversion': '9.7.2',
            'wm_visitid': '98d890a8-2f45-4e86-b2b5-ae83fa4976a2',
            'req_time': '1734944172660',
            'waimai_sign': '/',
            'wm_longitude': '108374982',
            'wm_latitude': '22766752',
            'wm_actual_longitude': '0',
            'wm_actual_latitude': '0',
            'lch': '1000',
            'utm_medium': 'android',
            'package_type': 'web',
            'wm_order_channel': 'md_h5',
            'business_entry_channel': 'default',
            'nickName': 'Joey2momo',
            'uuid': '97CAA3F4C5274B58E0BC0D926FF86136597DAE423E2B22E6A57436A9DE1093BA',
            'rc_platform': '3',
            'platform': '3',
            'rc_app': '82',
            'partner': '214',
            'riskLevel': '71',
            'client_id': '38',
            'biz_id': '1137',
            'wm_uuid': '97CAA3F4C5274B58E0BC0D926FF86136597DAE423E2B22E6A57436A9DE1093BA',
            'wm_poi_id': '-100',
            'uuid': '193f17e9ebac8-7e05714beb3c44-0-0-193f17e9ebac8',
            'rc_platform': '13',
            'platform': '13',
            'rc_app': '0',
            'partner': '214',
            'riskLevel': '71',
            'client_id': '90',
            'biz_id': '1137',
            'wm_poi_id': '-100',
            'poi_id_str': 'hC37xhceS5cgg3ORgGqy3QI',
            'spu_tag_id': '250917137_27',
            'tag_type': '27',
            'page_index': '0',
            'isFromShareCardForFanpa': 'false',
            'wm_ctype': 'sg_wxapp'
        }


        self.headers_wx = {
            "Host": "wx-shangou.meituan.com",
            "M-TRACEID": "-4535490894442474360",
            "csecuuid": "193f17e9ebac8-7e05714beb3c44-0-0-193f17e9ebac8",
            "csecuserid": "582315343",
            "Content-Type": "application/x-www-form-urlencoded",
            "Mtgsig": '{"a1":"1.2","a2":1735007086938,"a3":"ww703v76769x5v44zvyw14y2z8003y6z806u82vz8w797978822z6w17","a4":"c5c7fa26d02d9c7826fac7c5789c2dd07cd3bc33044f91e3","a5":"K1/z2PqLfKK92USyHHUnlsZ7U11rnCZUptXMUNKNQ6cu1nZCW5LMw+HzCWCe5noEypPp84k77MwsYlY9KhbgvgNbAqtQlpYQRpsjtg39UWEbXCMBvbQLwNumZcxj3+pk0qH7A9CPJjihpfus2An8qJ2n8rLtj4NMs/L98xS1p2POvQEBb7HikBbR5Wt201G5360J5gwHGmIz8IYMnkpa/9NcvLq57tNT/usMf2EdQgwl7KhmiPP=","a6":"w1.3r8F+8MAmLFPCEm4z+uU77C/tKC1KTleTlhdtBi7fVjItSXlMC6RdveukvY/kCmsQRoBTXB8jtcngv7ORi7U/Q9SI8Faozi+1iuSO6ywOAfgIcE0oSvHBllY9tzhWnUzj3MsDM5tc4VGwG2GzbTnODQfJeyT9xN4U42IcvvdfmwewPLUluyGm4ghNOpt4nyT4ySYrBoftsje56/5sj6GIIz6XBTdxpHC3qkOiC7xEFsi5QEncPQObdErLF1oIwCr3txLAzE7rdzegjRYQFEyitJiWVTcfSrk3XxsQIf01589qMJ/gwhNgGCSeoSDsMRvls82/VQW1KmQd6H7XsLbgrNTN3TEWP1v+uJohjEK1EE/m/0sumswptQi96VFl61Rr4tIgzRvpLMgr+Q3xAy1FDno9DzGOamHQYEp2NzavXGF1Oiic/jJmkqKKA3CD2ek6gRrjIPcnN9MxgoDxyEtzbx02agvklGdOoWhqCKx04Tn3wyVp7z7Ht0tYSWBBrmxx","a7":"wxde8ac0a21135c07d","x0":3,"d1":"cfa3104bc35d4040f678cf9ea8d7e044"}',
            "openIdCipher": "AwQAAABJAgAAAAEAAAAyAAAAPLgC95WH3MyqngAoyM/hf1hEoKrGdo0pJ5DI44e1wGF9AT3PH7Wes03actC2n/GVnwfURonD78PewMUppAAAADgMjEcsVvwM/urS3M3qKfZ8BYipYf/a6wQzzQVsZPYiKGJc0GfEgF9KW+UErdA8BQJMpqw7yZBcIw==",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c11)XWEB/11581",
            "xweb_xhr": "1",
            "openId": "oJVP50HoiBGl8iasWHvcQadCHUfE",
            "M-APPKEY": "wxmp_mt-weapp",
            "uuid": "193f17e9ebac8-7e05714beb3c44-0-0-193f17e9ebac8",
            "wm-ctype": "mt_weapp",
            "token": "AgEjIlQATekf5Kd00wEXtQ_HvKe36ZfGCQUCHdXpQ99QiL0cyvYMeWecwvo4SzyPmmosU7K54VHo9AAAAABwJQAApHLgUknqztFRYBrfDh8Kqib-Exv0d2d3_zxDYY2xD1bOsCzQAKolQCQ9n8nE_-VD",
        }

        self.data_wx = {
            'content_personalized_switch': '0',
            'ad_personalized_switch': '0',
            'wm_dtype': 'microsoft',
            'wm_dversion': '3.9.12',
            'wm_uuid': '193f17e9ebac8-7e05714beb3c44-0-0-193f17e9ebac8',
            'wm_visitid': 'c1741011-20ab-4355-95bf-ebaa1fc4048c',
            'wm_appversion': '8.51.2',
            'wm_logintoken': 'AgEjIlQATekf5Kd00wEXtQ_HvKe36ZfGCQUCHdXpQ99QiL0cyvYMeWecwvo4SzyPmmosU7K54VHo9AAAAABwJQAApHLgUknqztFRYBrfDh8Kqib-Exv0d2d3_zxDYY2xD1bOsCzQAKolQCQ9n8nE_-VD',
            'userToken': 'AgEjIlQATekf5Kd00wEXtQ_HvKe36ZfGCQUCHdXpQ99QiL0cyvYMeWecwvo4SzyPmmosU7K54VHo9AAAAABwJQAApHLgUknqztFRYBrfDh8Kqib-Exv0d2d3_zxDYY2xD1bOsCzQAKolQCQ9n8nE_-VD',
            'req_time': '1735007086976',
            'waimai_sign': '/',
            'wm_longitude': '108324810',
            'wm_latitude': '22753403',
            'longitude': '108324810',
            'latitude': '22753403',
            'wm_actual_longitude': '108320824',
            'wm_actual_latitude': '22756189',
            'userid': '582315343',
            'user_id': '582315343',
            'wx_union_id': 'oNQu9t71fhDX_WbwCEpg8BD3Q-aA',
            'city': '20',
            'open_id': 'oJVP50HoiBGl8iasWHvcQadCHUfE',
            'openidcipher': 'AwQAAABJAgAAAAEAAAAyAAAAPLgC95WH3MyqngAoyM/hf1hEoKrGdo0pJ5DI44e1wGF9AT3PH7Wes03actC2n/GVnwfURonD78PewMUppAAAADgMjEcsVvwM/urS3M3qKfZ8BYipYf/a6wQzzQVsZPYiKGJc0GfEgF9KW+UErdA8BQJMpqw7yZBcIw==',
            'uuid': '193f17e9ebac8-7e05714beb3c44-0-0-193f17e9ebac8',
            'rc_platform': '13',
            'platform': '13',
            'rc_app': '0',
            'partner': '214',
            'riskLevel': '71',
            'client_id': '90',
            'biz_id': '1137',
            'wm_poi_id': '-100',
            'poi_id_str': 'wKv271fA72Qn1p72p3Dq_wI',
            'spu_tag_id': '481540693_27',
            'tag_type': '27',
            'page_index': '0',
            'isFromShareCardForFanpa': 'false',
            'wm_ctype': 'mt_weapp'
        }

        
    # 获取需要爬取的url列表
    def get_url_list(self,url,total_nums):
        url_temp = url+'pn{}/'
        # 每一页显示显示15个美食  通过获取到每个分类下的总美食数来求出总页数
        pages = total_nums//15+1 if total_nums%15!=0 else total_nums//15
        url_list = [url_temp.format(i) for i in range(1,pages+1)]
        return url_list

    # 对url进行请求并返回处理后的响应信息
    def parse_url(self,url):
        time.sleep(1)
        rest = requests.get(url,headers=self.headers)
        print(rest);exit;
        html_str = re.findall(r'window._appState = (.*?);</script>', rest.content.decode())[0]
        return html_str

    # 创建item并进行存储
    def get_content_list(self,html_str,item):
        json_html = json.loads(html_str)
        foods = json_html['data']
        for i in foods:
            item['food_id'] = i['poiId']
            item['food_url'] = "https://www.meituan.com/meishi/{}/".format(item['food_id'])
            item['title'] = i['title']
            item['avg_score'] = i['avgScore']
            item['avg_price'] = i['avgPrice']
            item['comments'] = i['allCommentNum']
            item['area'] = i['address'][0:3]
            item['address'] = i['address']
            print(item)
            df = pandas.DataFrame(foods)
            df.to_csv('meituan.csv',index=False)

    # 主方法
    def run(self):
        # 首先请求入口url来获取每一个美食分类的url地址
        # 请看图例一
        # html_str = requests.get(self.start_url,headers=self.headers)
        # url = 'https://www.baidu.com/'
        # html_str = requests.get(url)

        html_str = requests.post(url=self.start_url,data=self.data_wx,headers=self.headers_wx)
        print(html_str.text);sys.exit()
        # 代码已经改变
        # html_str = html.etree.HTML(html_str.content.decode())
        # cate_list = html_str.xpath('//div[text()="分类"]/../ul/li')[1:]
        str_html = re.findall(r'window._appState = (.*?);</script>',html_str.content.decode())[0]
        json_html = json.loads(str_html)
        cate_list = json_html['filters']['cates'][1:]
        item_list = []

        # 对每一个分类进行分组分别获取美食的分类名和美食的分类的url
        for i in cate_list:
            item = {}
            # 分类的url进行反爬处理
            # 从网页中获取的url地址为 http://wx.meituan.com/meishi/c11/
            # 实际url地址为 https://wx.meituan.com/meishi/c11/
            # 因此需要将http替换成https
            # cate_url= i.xpath('./a/@href')[0]
            cate_url = i['url']
            item['cate_url'] = cate_url.replace('http','https')
            # item['cate_name'] = i.xpath('./a/text()')[0]
            item['cate_name'] = i['name']
            item_list.append(item)
        # 对每一个美食分类的分类名和分类url地址进行遍历并分别进行处理
        for i in item_list:
            # 睡眠1秒防止被识别为网络爬虫
            time.sleep(1)
            rest = requests.get(i['cate_url'],headers = self.headers)
            str_html = rest.content.decode()
            str_html = re.findall(r'window._appState = (.*?);</script>', str_html)[0]
            json_html = json.loads(str_html)
            total_nums = json_html['poiLists']['totalCounts']
            url_list = self.get_url_list(i['cate_url'],total_nums)
            for url in url_list:
                list_html = self.parse_url(url)
                self.get_content_list(list_html,i)


if __name__ == '__main__':
    meituan = MeituanSpider()
    meituan.run()